# -*- coding: utf-8 -*-
# @Date    : 2016-10-19 10:58:25
# @Author  : fancy (fancy@thecover.cn)


import re
import os
import json
import time

from scrapy.spiders import Spider
from scrapy.selector import Selector


GET_FIRST = lambda x: x[0] if x else None


class WeiboHot(Spider):
    name = 'weibo'
    rth_p = re.compile(r'view\(({"pid":"pl_top_realtimehot.*?})\)')
    total_p = re.compile(r'view\(({"pid":"pl_top_total".*?})\)')
    re_p = re.compile(r'\d+')
    cur = time.localtime()
    start_urls = [
        'http://s.weibo.com/top/summary?cate=realtimehot',      # 实时热搜
        'http://s.weibo.com/top/summary?cate=total&key=all',    # 热点搜索
        'http://s.weibo.com/top/summary?cate=total&key=films',  # 潮流搜索
        'http://s.weibo.com/top/summary?cate=total&key=person', # 名人热搜
    ]
    type_map = {
        0: '实时热搜',
        1: '热点搜索',
        2: '潮流搜索',
        3: '名人热搜',
    }

    def parse(self, response):
        if 'realtimehot' in response.url:
            js = self.rth_p.findall(response.body)
        else:
            js = self.total_p.findall(response.body)
        data = json.loads(js[0])
        sel = Selector(text=data['html'])
        hot_list = []
        ll = sel.xpath('body/div/table/tr')
        for item in ll:
            hot = {}
            # 排名
            hot['rank'] = int(item.xpath('td[1]/span/em/text()').extract()[0])
            # 关键词
            hot['content'] = item.xpath('td[2]/div/p/a/text()').extract()[0]
            hot['icon'] = GET_FIRST(item.xpath('td[2]/div/p/i/text()').extract())
            # 搜索指数
            hot['start_num'] = int(item.xpath('td[3]/p/span/text()').extract()[0])
            # 搜索热度
            redu = item.xpath('td[4]/p/span/@style').extract()[0]
            hot['rank_long'] = int(self.re_p.findall(redu)[0])
            hot_list.append(hot)
        self.save_data(hot_list, response.url)


    def save_data(self, data, url):
        index = self.start_urls.index(url)
        s1, s2 = self.now()
        dire = 'result/%s/%s/' % (s1, s2)
        file_name = '%s/%s.json' % (dire, self.type_map[index])
        self.find_or_create_dir(file_name)
        with open(file_name, 'w') as f:
            txt = json.dumps(data, ensure_ascii=False)
            f.write(txt)
        file_name1 = '%s/%s.txt' % (dire, self.type_map[index])
        with open(file_name1, 'w') as f:
            new_d = [json.dumps(i, ensure_ascii=False, sort_keys=True) for i in data]
            txt = '\n'.join(new_d)
            f.write(txt)


    @staticmethod
    def find_or_create_dir(file_name):
        directory = os.path.dirname(file_name)
        if not os.path.exists(directory):
            os.makedirs(directory)

    def now(self):
        day = time.strftime('%Y%m%d', self.cur)
        s = time.strftime('%H%M%S', self.cur)
        return day, s